#!/usr/bin/python
#from __future__ import print_function
import time
import sys

from google.appengine.ext import ndb
from google.appengine.ext.remote_api import remote_api_stub

ONE_MB = 1048576
BATCH_SIZE = 10000

def auth_func():
  return ('USER_NAME', 'PASSWORD')

remote_api_stub.ConfigureRemoteApi(None, '/_ah/remote_api', auth_func,
                                   'carlos-upload.appspot.com')
#Defines the entity model.
class Record(ndb.Model):
  key = ndb.StringProperty(indexed = True)
  lat = ndb.FloatProperty(indexed = False)
  lon = ndb.FloatProperty(indexed = False)

def main(argv):
  filename = argv[0]
  print "Processing: %s" % filename
  sys.stdout.flush()
  total_start = time.time()
  with open(filename) as f:
    count = 0
    records = [];
    # buffers the IO, reading line by line
    for row in f:
      values = row.strip('\n').split(';')
      assert len(values) == 3, "Invalid row: %s" % row
      records.append(Record(key = values[0],
                     lat = float(values[1]),
                     lon = float(values[2])))
      count += 1
      if len(records) == BATCH_SIZE :
        start = time.time()
        ndb.put_multi(records)
        records = []
        elapsed = time.time() - start
        totalel = time.time() - total_start
        print "    Submited: %9d - batch in %9ds - acc in %9ds)" % (count,
                                                                    elapsed,
                                                                    totalel)
        sys.stdout.flush()
    if len(records) > 0 :
      ndb.put_multi(records)
      records = []

    elapsed = time.time() - total_start
    print filename + ": %d registers processed in %ds.\n" % (count, elapsed)
    sys.stdout.flush()

if __name__ == "__main__":
     main(sys.argv[1:])
